{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Phase 5. Analysis of political social bots\n",
    "\n",
    "## Contents\n",
    "- [Configuration](#Configuration)\n",
    "  - [Imports](#Imports)\n",
    "  - [Variables](#Variables)\n",
    "  - [Support functions](#Support-functions)\n",
    "- [Cumulative bots' appearance](#Cumulative-bots'-appearance)\n",
    "  - [Appearence Graphs](#Appearence-Graphs)\n",
    "- [Bots daily activities](#Bots-daily-activities)\n",
    "  - [Active bots per day](#Active-bots-per-day)\n",
    "  - [Bots' activity windows](#Bots'-activity-windows)\n",
    "  - [Bots daily activity with cumulative metric](#Bots-daily-activity-with-cumulative-metric)\n",
    "- [Generated traffic](#Generated-traffic)\n",
    "  - [Bot's traffic volumes per political party](#Bot's-traffic-volumes-per-political-party)\n",
    "- [Sentiment analysis towards party themes](#Sentiment-analysis-towards-party-themes)\n",
    "  - [Sentiment score per political party](#Sentiment-score-per-political-party)\n",
    "  - [Sentiment score per political party (boxplot)](#Sentiment-score-per-political-party-(boxplot))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configuration\n",
    "\n",
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Utilities\n",
    "from IPython.display import display\n",
    "from fastprogress import master_bar, progress_bar\n",
    "import os\n",
    "from datetime import datetime\n",
    "from datetime import timedelta\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.options.display.max_columns = None\n",
    "pd.options.display.max_rows = None\n",
    "import csv\n",
    "from collections.abc import MutableMapping\n",
    "\n",
    "# Graphics\n",
    "from pandas.plotting import register_matplotlib_converters\n",
    "register_matplotlib_converters()\n",
    "import matplotlib.ticker as ticker\n",
    "import matplotlib.gridspec as gridspec\n",
    "import matplotlib.dates as mdates\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from matplotlib.dates import date2num\n",
    "\n",
    "\n",
    "# DB management\n",
    "from pymongo import MongoClient"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If true exports vectorial PDFs instead of JPG.\n",
    "VECTORIAL_FIGURES = True\n",
    "FIG_EXTENSION = \".pdf\" if VECTORIAL_FIGURES else \".jpg\"\n",
    "\n",
    "# Directories where CSV data is stored\n",
    "ROOT_DIR = \"ABOSLUTE_PATH_TO_ROOT_FOLDER\"\n",
    "DATA_DIR = ROOT_DIR + \"data/\"\n",
    "GRAPHICS_DIR = ROOT_DIR + \"graphics/\"\n",
    "\n",
    "# Change path to root\n",
    "os.chdir(ROOT_DIR)\n",
    "\n",
    "try:\n",
    "    os.makedirs(GRAPHICS_DIR)\n",
    "except FileExistsError:\n",
    "    # directory already exists\n",
    "    pass\n",
    "\n",
    "# MongoDB parameters\n",
    "mongoclient = MongoClient('IP_ADDRESS', PORT)\n",
    "db = mongoclient.DATABASE_NAME"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_objid(text):\n",
    "    \"\"\"Makes an ObjectId of 4 bytes\n",
    "    \n",
    "    Keyword arguments:\n",
    "    text -- string to be converted into Object ID\n",
    "    \"\"\"\n",
    "    if text is None:\n",
    "        return None\n",
    "    text = str(text)\n",
    "    if not text.strip():\n",
    "        return None\n",
    "    return ObjectId(text.rjust(24,\"0\"))\n",
    "\n",
    "\n",
    "def flatten(d, parent_key='', sep='_'):\n",
    "    \"\"\"Formats MongoDB results\n",
    "    \n",
    "    Keyword arguments:\n",
    "    d -- dictionary with key and uncleaned values\n",
    "    parent_key --\n",
    "    sep --\n",
    "    \"\"\"\n",
    "    items = []\n",
    "    for k, v in d.items():\n",
    "        new_key = parent_key + sep + k if parent_key else k\n",
    "        if isinstance(v, MutableMapping):\n",
    "            items.extend(flatten(v, new_key, sep=sep).items())\n",
    "        else:\n",
    "            items.append((new_key, v))\n",
    "    return dict(items)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cumulative bots' appearance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_first_appearances(user_collection):\n",
    "    \"\"\"Extracts a Dataframe with the dates of creation of the first tweet of each bot and associated political party.\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    \n",
    "    p95 = 0.6908019160064479\n",
    "    \n",
    "    pipeline = [\n",
    "    {\n",
    "        '$match': {\n",
    "            'scores.scores.universal': {\n",
    "                '$gte':p95\n",
    "            }\n",
    "        }\n",
    "    }, \n",
    "        {\n",
    "        '$lookup': {\n",
    "            'from': 'tweets', \n",
    "            'localField': '_id', \n",
    "            'foreignField': 'user_id', \n",
    "            'as': 'tweets'\n",
    "        }\n",
    "    }, {\n",
    "        '$unwind': {\n",
    "            'path': '$tweets'\n",
    "        }\n",
    "    }, {\n",
    "        '$sort': {\n",
    "            'tweets.created_at': 1\n",
    "        }\n",
    "    }, {\n",
    "        '$group': {\n",
    "            '_id': '$_id', \n",
    "            'created_at': {\n",
    "                '$first': '$tweets.created_at'\n",
    "            }, \n",
    "            'bot_political_party': {\n",
    "                '$first': '$bot_political_party'\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "    ]\n",
    "    \n",
    "    print(\"Query\", end=\" \")\n",
    "    bot_appearances = user_collection.aggregate(pipeline, allowDiskUse=True)\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    bot_appearances = list(bot_appearances)\n",
    "    print(\"OK; Total records:\", len(bot_appearances))\n",
    "    return pd.DataFrame(bot_appearances)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "df = get_first_appearances(db.users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.bot_political_party.fillna('Unknown',inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Appearence Graphs\n",
    "\n",
    "Generate three graphs:\n",
    "1. all bots, \n",
    "2. bots with two political parties, \n",
    "3. or bots with single political party (paper version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_dataframe(data,version):\n",
    "    \"\"\"Preprocesses and filters dataframe according to version of appearance. \n",
    "    \n",
    "    Keyword arguments:\n",
    "    data -- DataFrame with dates of creation of the first tweet of each bot and associated political party.\n",
    "    version -- possible values: 'all' (no filtering, political parties are classified as 'Unknown', 'Unclear', 'Two-party affinity' or 'One-party affinity','two parties')\n",
    "                                'two parties' (filters only bots with two-party affinity, grouping partyA-partyB and partyB-partyA in the same group)\n",
    "                                'one party' (filters only bots with one-party affinity)\n",
    "    \"\"\"\n",
    "    if version == 'all':\n",
    "        data['bot_political_party'] = df.bot_political_party.replace({'Unknown':'Unknown', # bots not labeled: insufficient information\n",
    "                                                'Unclear':'Unclear', # bots labeled as unclear: not able to categorize in one or two parties\n",
    "                                                # bots labeled with two parties\n",
    "                                                'VOX-Ciudadanos':'Two-party affinity', \n",
    "                                                'Ciudadanos-VOX':'Two-party affinity', \n",
    "                                                'PSOE-UP':'Two-party affinity',\n",
    "                                                'UP-PSOE':'Two-party affinity',\n",
    "                                                'VOX-UP':'Two-party affinity',\n",
    "                                                'UP-VOX':'Two-party affinity',        \n",
    "                                                'UP-Ciudadanos':'Two-party affinity',\n",
    "                                                'Ciudadanos-UP':'Two-party affinity',\n",
    "                                                'VOX-PP':'Two-party affinity',\n",
    "                                                'PP-VOX':'Two-party affinity',\n",
    "                                                'UP-PP':'Two-party affinity',\n",
    "                                                'PP-UP':'Two-party affinity',\n",
    "                                                'Ciudadanos-PSOE':'Two-party affinity',\n",
    "                                                'PSOE-Ciudadanos':'Two-party affinity',\n",
    "                                                'Ciudadanos-PP':'Two-party affinity',\n",
    "                                                'PP-Ciudadanos':'Two-party affinity',\n",
    "                                                'PSOE-VOX':'Two-party affinity',\n",
    "                                                'VOX-PSOE':'Two-party affinity', \n",
    "                                                'PSOE-PP':'Two-party affinity',\n",
    "                                                'PP-PSOE':'Two-party affinity',\n",
    "                                                # bots labelled with two parties,\n",
    "                                                'Ciudadanos':'One-party affinity',\n",
    "                                                'VOX':'One-party affinity',\n",
    "                                                'UP':'One-party affinity',\n",
    "                                                'PSOE':'One-party affinity',\n",
    "                                                'PP':'One-party affinity',\n",
    "                                            })\n",
    "        \n",
    "    elif version == 'two_parties':\n",
    "        data = data[data.bot_political_party.isin(['UP-VOX', 'Ciudadanos-UP','UP-Ciudadanos', 'VOX-Ciudadanos', 'PSOE-UP', 'UP-PP',\n",
    "                                                   'VOX-UP', 'Ciudadanos-VOX', 'Ciudadanos-PSOE', 'PP-UP', 'PP-Ciudadanos', 'VOX-PP',\n",
    "                                                   'UP-PSOE', 'PSOE-Ciudadanos', 'PSOE-VOX','Ciudadanos-PP', 'PP-VOX', 'VOX-PSOE', \n",
    "                                                   'PP-PSOE', 'PSOE-PP'])]\n",
    "        \n",
    "        data['bot_political_party'] = data.bot_political_party.replace({'VOX-Ciudadanos':'VOX-Ciudadanos',\n",
    "                                                                       'Ciudadanos-VOX':'VOX-Ciudadanos',\n",
    "                                                                       'PSOE-UP': 'UP-PSOE',\n",
    "                                                                       'UP-PSOE': 'UP-PSOE',\n",
    "                                                                       'VOX-UP': 'VOX-UP',\n",
    "                                                                       'UP-VOX': 'VOX-UP',        \n",
    "                                                                       'UP-Ciudadanos': 'UP-Ciudadanos',\n",
    "                                                                       'Ciudadanos-UP': 'UP-Ciudadanos',\n",
    "                                                                       'VOX-PP':'VOX-PP',\n",
    "                                                                       'PP-VOX':'VOX-PP',\n",
    "                                                                       'UP-PP':'UP-PP',\n",
    "                                                                       'PP-UP':'UP-PP',\n",
    "                                                                       'Ciudadanos-PSOE':'PSOE-Ciudadanos',\n",
    "                                                                       'PSOE-Ciudadanos':'PSOE-Ciudadanos',\n",
    "                                                                       'Ciudadanos-PP':'PP-Ciudadanos',\n",
    "                                                                       'PP-Ciudadanos':'PP-Ciudadanos',\n",
    "                                                                       'PSOE-VOX':'VOX-PSOE',\n",
    "                                                                       'VOX-PSOE':'VOX-PSOE', \n",
    "                                                                       'PSOE-PP':'PSOE-PP',\n",
    "                                                                       'PP-PSOE':'PSOE-PP'})  \n",
    "    else:\n",
    "        data = data[data.bot_political_party.isin(['UP','VOX','PSOE','PP','Ciudadanos'])]\n",
    "        data['bot_political_party'] = data.bot_political_party.replace({'Ciudadanos':'CS'})\n",
    "    \n",
    "    return data\n",
    "\n",
    "def zfil(text):\n",
    "    \"\"\"Auxiliar function to pad string on the left with zeros to fill a width of 2.\n",
    "    \n",
    "    Keyword arguments:\n",
    "    text -- string to be padded\n",
    "    \"\"\"\n",
    "    return text.zfill(2)\n",
    "\n",
    "\n",
    "def calculate_appearances(data):\n",
    "    \"\"\"Generates an ordered dataframe by date, containing political party occurrences within each date\n",
    "    \n",
    "    Keyword arguments:\n",
    "    data -- DataFrame with dates of creation of the first tweet of each bot and associated political party.\n",
    "    \"\"\"\n",
    "    \n",
    "    # create day and month columns from created_at\n",
    "    data['day'] = data['created_at'].dt.day\n",
    "    data['month'] = data['created_at'].dt.month\n",
    "    #data.drop(['_id','created_at'],axis='columns',inplace=True)\n",
    "    \n",
    "    # calculate ocurrences by month, day and political party\n",
    "    ocurrences = data.groupby(by=['month','day','bot_political_party']).size()\n",
    "    ocurrences = ocurrences.reset_index()\n",
    "    ocurrences.columns = ['month','day','political party','count']\n",
    "    \n",
    "    # force days to have two digits\n",
    "    ocurrences['day'] = ocurrences['day'].astype(str)\n",
    "    ocurrences['day'] = ocurrences['day'].apply(zfil)\n",
    "    \n",
    "    # sort rows by date\n",
    "    ocurrences['orden'] = ocurrences['month'].astype(str)  + ocurrences['day'].astype(str)\n",
    "    ocurrences['orden'] = ocurrences['orden'].astype(int)\n",
    "    ocurrences.sort_values(['orden','political party'],ascending=[True,True])\n",
    "    ocurrences['date'] = ocurrences['month'].astype(str) + \"-\" + ocurrences['day'].astype(str)\n",
    "    ocurrences.drop(['month','day'],axis='columns',inplace=True)\n",
    "    \n",
    "    return ocurrences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get colors for graphic\n",
    "party_colors = {\n",
    "    # one_party_version\n",
    "    'Ciudadanos':'#fa5000',\n",
    "    'PP':'#0bb2ff',\n",
    "    'PSOE':'#f41c14',\n",
    "    'UP':'#6b2d64',\n",
    "    'VOX':'#7cbd2a',\n",
    "    # all\n",
    "    'Unknown':'#3c4245',\n",
    "    'Unclear':'#719192',\n",
    "    'Two-party affinity':'#fda50f',\n",
    "    'One-party affinity':'#3f0403',\n",
    "    # two_parties_version\n",
    "    'VOX-Ciudadanos':'#bb8715',\n",
    "    'UP-PSOE':'#b0243c',\n",
    "    'VOX-UP':'#747547',\n",
    "    'UP-Ciudadanos':'#b33f32',\n",
    "    'VOX-PP':'#43b795',\n",
    "    'UP-PP':'#3b6fb2',\n",
    "    'PSOE-Ciudadanos':'#f7360a',\n",
    "    'PP-Ciudadanos':'#828180',\n",
    "    'VOX-PSOE':'#b86d1f',\n",
    "    'PSOE-PP':'#80678a'\n",
    "}\n",
    "\n",
    "\n",
    "# drawing three versions of accumulative appearance of bots\n",
    "for version in ['all','two_parties','one_party']:\n",
    "    data = df.copy()\n",
    "    data = preprocess_dataframe(data,version)\n",
    "    print(version,\"; number of bots:\",len(data),\"; possible classes:\",len(data['bot_political_party'].unique()))\n",
    "    \n",
    "    # preprocess_columns\n",
    "    data = calculate_appearances(data)\n",
    "    \n",
    "    # keep labels with dates\n",
    "    dates = []\n",
    "    for date in data['date'].astype(str).unique():\n",
    "        dates.append(date)\n",
    "    \n",
    "    # keep labels with political parties\n",
    "    parties = []\n",
    "    for party in data['political party'].unique():\n",
    "        parties.append(party)\n",
    "    \n",
    "    # create a list of lists: one per political party ocurrences by date\n",
    "    parties_ocurrences = []\n",
    "    for party in parties:\n",
    "        count = []\n",
    "        df_party = data[data['political party']==party]\n",
    "        for index,date in enumerate(dates):\n",
    "            political_party_date_row = df_party[df_party['date']==date]\n",
    "            \n",
    "            # acumulative count of users per date\n",
    "            exist_date = political_party_date_row.shape[0]==1\n",
    "            if exist_date:\n",
    "                if index==0:\n",
    "                    count.append(political_party_date_row['count'].iloc[0])\n",
    "                else:\n",
    "                    count.append(count[index-1]+political_party_date_row['count'].iloc[0])\n",
    "            else:\n",
    "                if index==0:\n",
    "                    count.append(0)\n",
    "                else:\n",
    "                    count.append(count[index-1])\n",
    "                    \n",
    "        parties_ocurrences.append(count)\n",
    "        \n",
    "  \n",
    "    colors = []\n",
    "    for party in parties:\n",
    "        colors.append(party_colors[party])\n",
    "    \n",
    "    # Draw graphics\n",
    "    plt.figure(figsize=(12,3))\n",
    "    plt.xticks(rotation='vertical')\n",
    "    plt.stackplot(dates,parties_ocurrences,labels=parties,colors=colors,edgecolor='black')\n",
    "    plt.title(\"Cumulative distribution of bots according to the first day of detection (\"+version+\")\")\n",
    "    plt.legend(loc='upper left')\n",
    "    plt.savefig(GRAPHICS_DIR+version+\"_cumulative_distributions_bots_first_detection.pdf\",bbox_inches = \"tight\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bots daily activities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_bot_users(user_collection):\n",
    "    \"\"\"Extracts the ObjectID and political party of bots\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    bot_users = user_collection.find({'bot_political_party':{'$exists':True}},\n",
    "                                  {'_id':1,'bot_political_party':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    bot_users = list(bot_users)\n",
    "    print(\"OK; Total bot users:\", len(bot_users))\n",
    "    return bot_users\n",
    "\n",
    "def load_tweets(tweet_collection):\n",
    "    \"\"\"Extracts the ObjectID, date of creation and user id of tweets\n",
    "    \n",
    "    Keyword arguments:\n",
    "    tweet_collection -- MongoDB Tweets' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    tweets = tweet_collection.find({},\n",
    "                            {'_id':1,'created_at':1,'user_id':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    tweets = list(tweets)\n",
    "    print(\"OK; Total tweets:\", len(tweets))\n",
    "    return tweets\n",
    "\n",
    "def reindex_by_date(df):\n",
    "    \"\"\"Reindexes a DataFrame by date\n",
    "    \n",
    "    Keyword arguments:\n",
    "    df -- DataFrame to be reindexed by date\n",
    "    \"\"\"\n",
    "    dates = pd.date_range(xlim[0], xlim[1],name='created_at')\n",
    "    return df.reindex(dates,fill_value=0)\n",
    "\n",
    "def daterange(start_date, end_date):\n",
    "    \"\"\"Creates a generator of days to be used in loops\n",
    "    \n",
    "    Keyword arguments:\n",
    "    start_date -- First date to be generated\n",
    "    end_date -- Last date to be generated\n",
    "    \"\"\"\n",
    "    for n in range(int ((end_date - start_date).days)):\n",
    "        yield start_date + timedelta(n)\n",
    "\n",
    "# oficial party colors\n",
    "party_colors = {\n",
    "    'Ciudadanos':'#fa5000', 'CS':'#fa5000',\n",
    "    'PP':'#0bb2ff',\n",
    "    'PSOE':'#f41c14',\n",
    "    'UP':'#6b2d64',\n",
    "    'VOX':'#7cbd2a'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bot_users = load_bot_users(db.users)\n",
    "df_bot_users = pd.DataFrame(bot_users)\n",
    "df_bot_users = df_bot_users[df_bot_users.bot_political_party.isin(['VOX','Ciudadanos','PP','PSOE','UP'])]\n",
    "df_bot_users.columns = ['_id','political_party']\n",
    "display(df_bot_users.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "tweets = load_tweets(db.tweets)\n",
    "tweets = [flatten(t) for t in tweets]\n",
    "df_tweets = pd.DataFrame(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "df_bot_tweets = df_tweets[df_tweets['user_id'].isin(df_bot_users['_id'])]\n",
    "df_bot_tweets = df_bot_tweets.join(df_bot_users.set_index('_id'), on='user_id') ## append political party to interaction\n",
    "df_bot_tweets.dropna(axis='index',inplace=True)\n",
    "df_bot_tweets.created_at = df_bot_tweets.created_at.astype(\"datetime64\")\n",
    "display(df_bot_tweets.head(2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Active bots per day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# limits of x-axis\n",
    "xlim = (df_bot_tweets.created_at.dt.date.min(), df_bot_tweets.created_at.dt.date.max())\n",
    "\n",
    "# Extract number of active bots per date and political party\n",
    "data = df_bot_tweets.copy()\n",
    "data['date'] = data.created_at.dt.date\n",
    "data.set_index(data.created_at, drop=True, inplace=True)\n",
    "data = data.groupby(['date','political_party']).agg({'user_id':pd.Series.nunique})\n",
    "data = pd.DataFrame(data.to_records())\n",
    "data.columns = ['Date','Political party','Active bots']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot\n",
    "plt.figure(figsize=(15,5))\n",
    "ax = sns.lineplot(data=data, x='Date', y='Active bots', hue='Political party', hue_order=['UP','PSOE','Ciudadanos','PP','VOX'],palette=party_colors)\n",
    "plt.legend(loc='upper left')\n",
    "plt.title('Bot activity')\n",
    "plt.xlim(left=xlim[0], right=xlim[1])\n",
    "ax.set_ylabel(\"Number of active bots\")\n",
    "ax.xaxis.set_major_locator(mdates.DayLocator())\n",
    "ax.xaxis.set_tick_params(rotation=75)\n",
    "#plt.savefig(GRAPHICS_DIR+\"timeline/\"+version+\"-traffic-timeline-original.pdf\", bbox_inches = \"tight\")\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bots' activity windows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the first day and last day of activity of bots\n",
    "data2 = df_bot_tweets.copy()\n",
    "display(data2.head())\n",
    "data2 = data2.groupby(['user_id','political_party']).agg({'created_at':['min','max']})\n",
    "data2 = pd.DataFrame(data2.to_records())\n",
    "data2.columns = ['user_id','political_party','min','max']\n",
    "data2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform timestamp y date\n",
    "data2['min']= data2['min'].dt.date\n",
    "data2['max'] = data2['max'].dt.date\n",
    "display(data2.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get start date and end date from dataframe\n",
    "start = min(data2['min'])\n",
    "end = max(data2['max'])\n",
    "\n",
    "# initializes a dictionary of dates\n",
    "activity = {}\n",
    "for d in daterange(start,end+timedelta(1)):\n",
    "    # each date has another dictionary with each political party\n",
    "    activity[d] = {}\n",
    "    for party,party_group in data2.groupby('political_party'):\n",
    "        activity[d][party] = 0\n",
    "\n",
    "# the dictionary activity is filled with active users\n",
    "# in this case, user activity is the range from the date of the first iteration (min_date) and the last iteration (max_date)\n",
    "# a bot is considered to be active in a specific day d if min_date <= d <= max_date\n",
    "for d in daterange(start,end+timedelta(1)):\n",
    "    for party,party_group in data2.groupby('political_party'):\n",
    "        for min_date, max_date in zip(party_group['min'],party_group['max']):\n",
    "            if min_date <= d and d <= max_date:\n",
    "                activity[d][party] = activity[d][party]+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# activity dictionary is converted into a DataFrame with active bots per date and political party\n",
    "df = pd.DataFrame.from_dict(data=activity,orient='index')\n",
    "df.reset_index(inplace=True)\n",
    "df = df.melt(id_vars=['index'],value_vars=['Ciudadanos','PP','PSOE','UP','VOX'])\n",
    "df.columns = ['Date','Political party','Active Social Bots']\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot\n",
    "plt.figure(figsize=(15,5))\n",
    "w = -0.3\n",
    "\n",
    "# bot daily activity in barplots\n",
    "for party in ['UP','PSOE','Ciudadanos','PP','VOX']:\n",
    "    plt.bar(date2num(data[data['Political party']==party]['Date'])+w,height=data[data['Political party']==party]['Active bots'],color=party_colors[party],width=0.15,align='center')\n",
    "    w=0.15+w\n",
    "    \n",
    "# bot range activity in lineplots\n",
    "ax = sns.lineplot(data=df, x='Date', y='Active Social Bots', hue='Political party', hue_order=['UP','PSOE','Ciudadanos','PP','VOX'], palette=party_colors)\n",
    "plt.legend(loc='upper left')\n",
    "plt.title('Bot activity')\n",
    "plt.xlim(left=xlim[0], right=xlim[1])\n",
    "ax.set_ylabel(\"Number of active bots\")\n",
    "ax.xaxis.set_major_locator(mdates.DayLocator())\n",
    "ax.xaxis.set_tick_params(rotation=75)\n",
    "#plt.savefig(GRAPHICS_DIR+\"timeline/\"+version+\"-traffic-timeline-original.pdf\", bbox_inches = \"tight\")\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bots daily activity with cumulative metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activity per political party and date\n",
    "df_activity_per_day = df_bot_tweets.copy()\n",
    "df_activity_per_day['date'] = df_activity_per_day.created_at.dt.date\n",
    "df_activity_per_day.set_index(df_activity_per_day.created_at, drop=True, inplace=True)\n",
    "df_activity_per_day = df_activity_per_day.groupby(['date','political_party']).agg({'user_id':pd.Series.nunique})\n",
    "df_activity_per_day = pd.DataFrame(df_activity_per_day.to_records())\n",
    "df_activity_per_day.columns = ['Date','Political party','Active Social Bots']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cumulative activity per political party and date\n",
    "df_cumulative = df_bot_tweets.copy()\n",
    "df_cumulative['created_at']= df_cumulative['created_at'].dt.date\n",
    "start = min(df_cumulative['created_at'])\n",
    "end = max(df_cumulative['created_at'])\n",
    "print(start,end)\n",
    "df_cumulative = df_cumulative.groupby(['user_id','political_party']).agg({'created_at':'min'})\n",
    "df_cumulative = pd.DataFrame(df_cumulative.to_records())\n",
    "df_cumulative.columns = ['user_id','political_party','min']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cumulative dictionary initialization\n",
    "cumulative = {}\n",
    "for d in daterange(start,end+timedelta(1)):\n",
    "    cumulative[d] = {}\n",
    "    for party,party_group in df_cumulative.groupby('political_party'):\n",
    "        cumulative[d][party] = 0\n",
    "\n",
    "# the dictionary cumulative is filled with active users\n",
    "# in cumulative activity, the presence of a bot is considere from its first iteration (min_date)\n",
    "# a bot is considered to be active in a specific day d if min_date <= d\n",
    "for d in daterange(start,end+timedelta(1)):\n",
    "    for party,party_group in df_cumulative.groupby('political_party'):\n",
    "        if d>start:\n",
    "            cumulative[d][party] = cumulative[d-timedelta(1)][party]    \n",
    "        for min_date in party_group['min']:\n",
    "            if min_date == d:\n",
    "                cumulative[d][party] = cumulative[d][party]+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cumulative dictionary is transformed into a dataframe with the cumulative bots activity per date and political party\n",
    "df2 = pd.DataFrame.from_dict(data=cumulative,orient='index')\n",
    "df2.reset_index(inplace=True)\n",
    "df2 = df2.melt(id_vars=['index'],value_vars=['Ciudadanos','PP','PSOE','UP','VOX'])\n",
    "df2.columns = ['Date','Political party','Cumulative Social Bots Activity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we replace the word 'Ciudadanos' with its acronym\n",
    "df2['Political party'].replace({'Ciudadanos':'CS'}, inplace=True)\n",
    "df_activity_per_day['Political party'].replace({'Ciudadanos':'CS'}, inplace=True)\n",
    "\n",
    "# plot bots activity per day in barplots\n",
    "plt.figure(figsize=(15,5))\n",
    "w = -0.3\n",
    "for party in ['UP','PSOE','CS','PP','VOX']:\n",
    "    plt.bar(date2num(df_activity_per_day[df_activity_per_day['Political party']==party]['Date'])+w,height=df_activity_per_day[df_activity_per_day['Political party']==party]['Active Social Bots'],color=party_colors[party],width=0.15,align='center')\n",
    "    w=0.15+w\n",
    "\n",
    "# plot bots accumulative activity per day in lineplots\n",
    "ax = sns.lineplot(data=df2, x='Date', y='Cumulative Social Bots Activity', hue='Political party', hue_order=['UP','PSOE','CS','PP','VOX'], palette=party_colors)\n",
    "plt.legend(loc='upper left')\n",
    "plt.title('Bot activity', **title_font)\n",
    "plt.xlim(left=start-timedelta(1), right=end+timedelta(1))\n",
    "ax.set_ylabel(\"Number of active bots\")\n",
    "ax.xaxis.set_major_locator(mdates.DayLocator())\n",
    "ax.xaxis.set_tick_params(rotation=75)\n",
    "plt.tight_layout()\n",
    "plt.savefig(GRAPHICS_DIR+\"activity-timeline.pdf\", bbox_inches = \"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generated traffic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bot_tweets(user_collection):\n",
    "    \"\"\"Extracts the ObjectID, user id, user's political party, tweet type, date of creation, retweet count and favourite count of bot interactions\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    p95 = 0.6908019160064479\n",
    "    \n",
    "    pipeline = [\n",
    "    {\n",
    "        '$match': {\n",
    "            'scores.scores.universal': {\n",
    "                '$gte': p95\n",
    "            }\n",
    "        }\n",
    "    }, {\n",
    "        '$lookup': {\n",
    "            'from': 'tweets', \n",
    "            'localField': '_id', \n",
    "            'foreignField': 'user_id', \n",
    "            'as': 'tweets'\n",
    "        }\n",
    "    }, {\n",
    "        '$unwind': {\n",
    "            'path': '$tweets'\n",
    "        }\n",
    "    }, {\n",
    "        '$project': {\n",
    "            '_id': 0, \n",
    "            'user_id': '$_id', \n",
    "            'bot_political_party': True, \n",
    "            'tweet_type': '$tweets.tweet_type', \n",
    "            'date': '$tweets.created_at', \n",
    "            'retweet_count': '$tweets.retweet_count', \n",
    "            'favorite_count': '$tweets.favorite_count'\n",
    "        }\n",
    "    }\n",
    "    ]\n",
    "    \n",
    "    print(\"Processing MONGO request\",end=\"; \")\n",
    "    tweets = user_collection.aggregate(pipeline)\n",
    "    print(\"OK\",end=\"; \")\n",
    "    tweets = list(tweets)\n",
    "    print(\"Length\",len(tweets),end=\";\")\n",
    "    return tweets\n",
    "\n",
    "def reindex_by_date(df):\n",
    "    \"\"\"Reindexes a DataFrame by date\n",
    "    \n",
    "    Keyword arguments:\n",
    "    df -- DataFrame to be reindexed by date\n",
    "    \"\"\"\n",
    "    dates = pd.date_range(xlim[0], xlim[1],name='created_at')\n",
    "    return df.reindex(dates,fill_value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bot_tweets = get_bot_tweets(db.users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(bot_tweets)\n",
    "df.date = df.date.astype(\"datetime64\")\n",
    "df.bot_political_party.fillna('Unknown',inplace=True)\n",
    "display(df.head(6))\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get limits of x axis\n",
    "xlim = (df.date.dt.date.min(), df.date.dt.date.max())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bot's traffic volumes per political party\n",
    "\n",
    "Draw three charts:\n",
    "1. all bots,\n",
    "2. bots with two political parties,\n",
    "3. bots with single political party (paper version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get colors for graphic\n",
    "party_colors = {\n",
    "    # one_party_version\n",
    "    'CS':'#fa5000',\n",
    "    'PP':'#0bb2ff',\n",
    "    'PSOE':'#f41c14',\n",
    "    'UP':'#6b2d64',\n",
    "    'VOX':'#7cbd2a',\n",
    "    # all\n",
    "    'Unknown':'#3c4245',\n",
    "    'Unclear':'#719192',\n",
    "    'Two-party affinity':'#fda50f',\n",
    "    'One-party affinity':'#3f0403',\n",
    "    # two_parties_version\n",
    "    'VOX-Ciudadanos':'#bb8715',\n",
    "    'UP-PSOE':'#b0243c',\n",
    "    'VOX-UP':'#747547',\n",
    "    'UP-Ciudadanos':'#b33f32',\n",
    "    'VOX-PP':'#43b795',\n",
    "    'UP-PP':'#3b6fb2',\n",
    "    'PSOE-Ciudadanos':'#f7360a',\n",
    "    'PP-Ciudadanos':'#828180',\n",
    "    'VOX-PSOE':'#b86d1f',\n",
    "    'PSOE-PP':'#80678a'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "plot three versions of generated traffic\n",
    "    all: no filtering, political parties are classified as 'Unknown', 'Unclear', 'Two-party affinity' or 'One-party affinity','two parties',\n",
    "    two parties: filters only bots with two-party affinity, grouping partyA-partyB and partyB-partyA in the same group\n",
    "    one party: filters only bots with one-party affinity\n",
    "'''\n",
    "for version in ['all','two_parties','one_party']:\n",
    "    data = df.copy()\n",
    "    data = preprocess_dataframe(data,version)\n",
    "\n",
    "    \n",
    "    plt.figure(figsize=(15,3))\n",
    "    for party,party_group in data.groupby(by=['bot_political_party']):\n",
    "        grp = party_group.copy()\n",
    "        grp = grp[grp.tweet_type.isin(['original','reply','quote'])]\n",
    "        grp.set_index(grp.date, drop=True, inplace=True)\n",
    "        k1 = grp.groupby(grp.date.dt.date).agg({'bot_political_party':'count', 'retweet_count':'sum'}).apply(reindex_by_date)\n",
    "        ax1 = sns.lineplot(data=k1, x=k1.index, y=k1.bot_political_party, label=party, color=party_colors[party])\n",
    "        \n",
    "    if version == 'one_party':\n",
    "        plt.legend(loc='upper left', title=\"Political party\")\n",
    "    else:\n",
    "        plt.legend(bbox_to_anchor=(1,1), title=\"Party\")\n",
    "    plt.title(version+\" \"+\"original, reply, quote\")\n",
    "    plt.xlim(left=xlim[0], right=xlim[1])\n",
    "    plt.tight_layout()\n",
    "    ax1.xaxis.set_major_locator(mdates.DayLocator())\n",
    "    ax1.xaxis.set_tick_params(rotation=75)\n",
    "    #plt.savefig(GRAPHICS_DIR+\"timeline/\"+version+\"-traffic-timeline-original.pdf\", bbox_inches = \"tight\")\n",
    "    plt.show()\n",
    "    \n",
    "    plt.figure(figsize=(15,3))\n",
    "    for party,party_group in data.groupby(by=['bot_political_party']):\n",
    "        grp = party_group.copy()\n",
    "        grp = grp[grp.tweet_type.isin(['original','reply','quote'])]\n",
    "        grp.set_index(grp.date, drop=True, inplace=True)\n",
    "        k1 = grp.groupby(grp.date.dt.date).agg({'bot_political_party':'count', 'retweet_count':'sum'}).apply(reindex_by_date)\n",
    "        ax2 = sns.lineplot(data=k1, x=k1.index, y=k1.retweet_count, label=party, color=party_colors[party])\n",
    "        ax2.fill_between(k1.index, 0, k1.retweet_count, color=party_colors[party], alpha=0.2)\n",
    "        \n",
    "    if version == 'one_party':\n",
    "        plt.legend(loc='upper left', title=\"Political party\")\n",
    "    else:\n",
    "        plt.legend(bbox_to_anchor=(1,1), title=\"Party\")\n",
    "        \n",
    "    plt.title(version+\" \"+\"associated retweets\")\n",
    "    plt.xlim(left=xlim[0], right=xlim[1])\n",
    "    plt.tight_layout()\n",
    "    ax2.xaxis.set_major_locator(mdates.DayLocator())\n",
    "    ax2.xaxis.set_tick_params(rotation=75)\n",
    "    #plt.savefig(GRAPHICS_DIR+\"timeline/\"+version+\"-traffic-timeline-associated-retweets.pdf\", bbox_inches = \"tight\")\n",
    "    plt.show()\n",
    "    \n",
    "    plt.figure(figsize=(15,3))\n",
    "    for party,party_group in data.groupby(by=['bot_political_party']):\n",
    "        grp2 = party_group.copy()\n",
    "        grp2 = grp2[grp2.tweet_type=='retweet']\n",
    "        grp2.set_index(grp2.date, drop=True, inplace=True)\n",
    "        k2 = grp2.groupby(grp2.date.dt.date).agg({'bot_political_party':'count'}).apply(reindex_by_date)\n",
    "        ax3 = sns.lineplot(data=k2, x=k2.index, y=k2.bot_political_party, label=party, color=party_colors[party])\n",
    "        \n",
    "    if version == 'one_party':\n",
    "        plt.legend(loc='upper left', title=\"Political party\")\n",
    "    else:\n",
    "        plt.legend(bbox_to_anchor=(1,1), title=\"Party\")\n",
    "        \n",
    "    plt.legend(bbox_to_anchor=(1,1), title=\"Party\")\n",
    "    plt.title(version+\" \"+\"total retweets\")\n",
    "    plt.xlim(left=xlim[0], right=xlim[1])\n",
    "    plt.tight_layout()\n",
    "    ax3.xaxis.set_major_locator(mdates.DayLocator())\n",
    "    ax3.xaxis.set_tick_params(rotation=75)\n",
    "    #plt.savefig(GRAPHICS_DIR+\"timeline/\"+version+\"-traffic-timeline-total-retweets.pdf\", bbox_inches = \"tight\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Pretty representation for PAPER\n",
    "data = df.copy()\n",
    "data = preprocess_dataframe(data,\"one_party\")\n",
    "plt.figure(figsize=(15,3))\n",
    "\n",
    "for party in ['UP','PSOE','CS','PP','VOX']:\n",
    "    grp = data[data.bot_political_party==party].copy()\n",
    "    grp = grp[grp.tweet_type.isin(['original','reply','quote'])]\n",
    "    grp.set_index(grp.date, drop=True, inplace=True)\n",
    "    k1 = grp.groupby(grp.date.dt.date).agg({'bot_political_party':'count', 'retweet_count':'sum'}).apply(reindex_by_date)\n",
    "    ax1 = sns.lineplot(data=k1, x=k1.index, y=k1.bot_political_party, label=party, color=party_colors[party])\n",
    "\n",
    "plt.legend(loc='upper left', title=\"Political party\")\n",
    "plt.title('Originals, quotes and replies')\n",
    "plt.ylabel(\"Number of interactions\")\n",
    "plt.xlabel(\"Date\")\n",
    "plt.xlim(left=xlim[0], right=xlim[1])\n",
    "plt.tight_layout()\n",
    "ax1.xaxis.set_major_locator(mdates.DayLocator())\n",
    "ax1.xaxis.set_tick_params(rotation=75)\n",
    "plt.savefig(GRAPHICS_DIR+\"traffic-timeline-1.pdf\", bbox_inches = \"tight\")\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=(15,3))\n",
    "for party in ['UP','PSOE','CS','PP','VOX']:\n",
    "    grp = data[data.bot_political_party==party].copy()\n",
    "    grp = grp[grp.tweet_type.isin(['original','reply','quote'])]\n",
    "    grp.set_index(grp.date, drop=True, inplace=True)\n",
    "    k1 = grp.groupby(grp.date.dt.date).agg({'bot_political_party':'count', 'retweet_count':'sum'}).apply(reindex_by_date)\n",
    "    ax2 = sns.lineplot(data=k1, x=k1.index, y=k1.retweet_count, label=party, color=party_colors[party])\n",
    "    ax2.fill_between(k1.index, 0, k1.retweet_count, color=party_colors[party], alpha=0.2)\n",
    "\n",
    "plt.legend(loc='upper left', title=\"Political party\")\n",
    "#plt.legend().set_visible(False)\n",
    "plt.title('Originals, quotes and replies\\' retweets')\n",
    "plt.ylabel(\"Number of associated retweets\")\n",
    "plt.xlim(left=xlim[0], right=xlim[1])\n",
    "plt.xlabel(\"Date\")\n",
    "plt.tight_layout()\n",
    "ax2.xaxis.set_major_locator(mdates.DayLocator())\n",
    "ax2.xaxis.set_tick_params(rotation=75)\n",
    "plt.savefig(GRAPHICS_DIR+\"traffic-timeline-2.pdf\", bbox_inches = \"tight\")\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=(15,3))\n",
    "for party in ['UP','PSOE','CS','PP','VOX']:\n",
    "    grp2 = data[data.bot_political_party==party].copy()\n",
    "    grp2 = grp2[grp2.tweet_type=='retweet']\n",
    "    grp2.set_index(grp2.date, drop=True, inplace=True)\n",
    "    k2 = grp2.groupby(grp2.date.dt.date).agg({'bot_political_party':'count'}).apply(reindex_by_date)\n",
    "    ax3 = sns.lineplot(data=k2, x=k2.index, y=k2.bot_political_party, label=party, color=party_colors[party])\n",
    "\n",
    "plt.legend(loc='upper left', title=\"Political party\")\n",
    "#plt.legend().set_visible(False)\n",
    "plt.title(\"Total retweets\")\n",
    "plt.xlim(left=xlim[0], right=xlim[1])\n",
    "plt.ylabel(\"Number of retweets\")\n",
    "plt.xlabel(\"Date\")\n",
    "plt.tight_layout()\n",
    "ax3.xaxis.set_major_locator(mdates.DayLocator())\n",
    "ax3.xaxis.set_tick_params(rotation=75)\n",
    "#plt.savefig(GRAPHICS_DIR+\"traffic-timeline-3.pdf\", bbox_inches = \"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sentiment analysis towards party themes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_labeled_users(user_collection):\n",
    "    \"\"\"Extracts the ObjectID and political party of manually labeled users\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    labeled_users = user_collection.find({'political_party':{'$exists':True}},\n",
    "                                  {'_id':1,'political_party':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    labeled_users = list(labeled_users)\n",
    "    print(\"OK; Total labeled users:\", len(labeled_users))\n",
    "    return labeled_users\n",
    "\n",
    "def load_bot_users(user_collection):\n",
    "    \"\"\"Extracts the ObjectID and political party of bots\n",
    "    \n",
    "    Keyword arguments:\n",
    "    user_collection -- MongoDB Users' Collection\n",
    "    \"\"\"\n",
    "    p95 = 0.6908019160064479\n",
    "    print(\"Query\", end=\" \")\n",
    "    bot_users = user_collection.find({'bot_political_party':{'$exists':True}},\n",
    "                                  {'_id':1,'bot_political_party':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    bot_users = list(bot_users)\n",
    "    print(\"OK; Total bot users:\", len(bot_users))\n",
    "    return bot_users\n",
    "\n",
    "def load_tweets(tweet_collection):\n",
    "    \"\"\"Extracts the ObjectID, tweet type, bag-of-words, sentiment score and user id of all tweets\n",
    "    \n",
    "    Keyword arguments:\n",
    "    tweet_collection -- MongoDB Tweets' Collection\n",
    "    \"\"\"\n",
    "    print(\"Query\", end=\" \")\n",
    "    tweets = tweet_collection.find({},\n",
    "                            {'_id':1,'tweet_type':1,'keywords_summary':1,'sentiment_score':1,'user_id':1})\n",
    "    print(\"OK; List\", end=\" \")\n",
    "    tweets = list(tweets)\n",
    "    print(\"OK; Total tweets:\", len(tweets))\n",
    "    return tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "labeled_users = load_labeled_users(db.users)\n",
    "df_labeled_users = pd.DataFrame(labeled_users)\n",
    "display(df_labeled_users.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bot_users = load_bot_users(db.users)\n",
    "df_bot_users = pd.DataFrame(bot_users)\n",
    "df_bot_users = df_bot_users[df_bot_users.bot_political_party.isin(['VOX','Ciudadanos','PP','PSOE','UP'])]\n",
    "df_bot_users.columns = ['_id','political_party']\n",
    "display(df_bot_users.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "tweets = load_tweets(db.tweets)\n",
    "tweets = [flatten(t) for t in tweets]\n",
    "df_tweets = pd.DataFrame(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# get tweets of manually labeled users by joining two DataFrames through 'user_id' key\n",
    "df_labeled_tweets = df_tweets[df_tweets['user_id'].isin(df_labeled_users['_id'])]\n",
    "df_labeled_tweets = df_labeled_tweets.join(df_labeled_users.set_index('_id'), on='user_id') ## append political party to interaction\n",
    "df_labeled_tweets.dropna(axis='index',inplace=True)\n",
    "display(df_labeled_tweets.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# get tweets of bot users by joining two DataFrames through 'user_id' key\n",
    "df_bot_tweets = df_tweets[df_tweets['user_id'].isin(df_bot_users['_id'])]\n",
    "df_bot_tweets = df_bot_tweets.join(df_bot_users.set_index('_id'), on='user_id') ## append political party to interaction\n",
    "df_bot_tweets.dropna(axis='index',inplace=True)\n",
    "display(df_bot_tweets.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parties = ['VOX','PP','Ciudadanos','PSOE','UP']\n",
    "colors ={'VOX':'#7cbd2a',\n",
    "        'PP':'#0bb2ff',\n",
    "        'Ciudadanos':'#fa5000',\n",
    "        'PSOE':'#f41c14',\n",
    "        'UP':'#6b2d64'}\n",
    "\n",
    "def filter_single_party_interactions(df):\n",
    "    '''\n",
    "    Filters the DataFrame to maintain only tweets dealing with a single political party. The mentioned party is inserted in a new column 'passive_political_party'\n",
    "    \n",
    "    Keyword arguments:\n",
    "    df -- DataFrame of tweets containing at least the ObjectID (_id) and keyword summaries of parties (keywords_summary_P, where P in parties)\n",
    "    '''\n",
    "    print(\"Dataframe with\",len(df),\"records\",end=\"; \")\n",
    "    for p in parties:\n",
    "        passive_political_party_group = df[df['keywords_summary_'+p]==True]\n",
    "        for p2 in parties:\n",
    "            if p2 is not p:\n",
    "                passive_political_party_group = passive_political_party_group[passive_political_party_group['keywords_summary_'+p2]==False]\n",
    "\n",
    "        idxs = passive_political_party_group.index.values\n",
    "\n",
    "        for idx in idxs:\n",
    "            df.at[idx,'passive_political_party'] = p\n",
    "    \n",
    "    df.dropna(axis='index',inplace=True)\n",
    "    print(\"Filtered dataframe with\",len(df),\"records\")\n",
    "    df['passive_political_party'] = df.passive_political_party.astype('category')\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# manually labeled sentiments\n",
    "data = filter_single_party_interactions(df_labeled_tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bots sentiments\n",
    "data2 = filter_single_party_interactions(df_bot_tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sentiment score per political party"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "make stripplot with originals, replies and quotes per political party and party theme for:\n",
    " - manually labeled users\n",
    " - bots\n",
    " \n",
    "In X-axis, the political party of users\n",
    "In Y-axis, the sentiment score of each tweet\n",
    "The color indicates which party the tweet is about\n",
    "'''\n",
    "for d in ['Manually labeled','Bot']:\n",
    "    if d=='Manually labeled':\n",
    "        paint = data\n",
    "    else:\n",
    "        paint = data2\n",
    "        \n",
    "    plt.figure(figsize=(15,4))\n",
    "    sns.stripplot(x=\"political_party\", \n",
    "                  y=\"sentiment_score\", \n",
    "                  hue=\"passive_political_party\", \n",
    "                  order=['UP','PSOE','Ciudadanos','PP','VOX'],\n",
    "                  hue_order=['UP','PSOE','Ciudadanos','PP','VOX'],\n",
    "                  palette=colors, \n",
    "                  data=paint[paint.tweet_type.isin(['original','reply','quote'])],\n",
    "                  jitter=0.25,\n",
    "                  dodge=True)\n",
    "    \n",
    "    plt.legend(bbox_to_anchor=(1,1), title=\"Party theme\")\n",
    "    plt.title(d+' users: generated originals, replies and quotes')\n",
    "    plt.xlabel(d+\" political party\")\n",
    "    plt.ylabel(\"Sentiment score\")\n",
    "    plt.savefig(GRAPHICS_DIR+d+\"-sentiment-distribution.pdf\", bbox_inches = \"tight\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sentiment score per political party (boxplot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "make boxplots of original, reply and quote sentiments per political party and party theme for:\n",
    " - manually labeled users\n",
    " - bots\n",
    " \n",
    "In X-axis, the political party of users\n",
    "In Y-axis, the sentiment score of each tweet\n",
    "The color indicates which party the tweet is about\n",
    "'''\n",
    "\n",
    "fig, axs = plt.subplots(nrows=2, ncols=1,figsize=(15,8))\n",
    "\n",
    "i=0\n",
    "for d in ['Manually labeled','Social Bots']:\n",
    "    if d=='Manually labeled':\n",
    "        paint = data\n",
    "        xlb =''\n",
    "        pad=30\n",
    "    else:\n",
    "        paint = data2\n",
    "        xlb = 'User\\'s political party'\n",
    "        pad=8\n",
    "    \n",
    "    paint.political_party.replace({'Ciudadanos':'CS'},inplace=True)\n",
    "    paint.passive_political_party.replace({'Ciudadanos':'CS'},inplace=True)\n",
    "\n",
    "    sns.boxplot(x=\"political_party\", \n",
    "                  y=\"sentiment_score\", \n",
    "                  hue=\"passive_political_party\", \n",
    "                  order=['UP','PSOE','CS','PP','VOX'],\n",
    "                  hue_order=['UP','PSOE','CS','PP','VOX'],\n",
    "                  palette=colors, \n",
    "                  data=paint[paint.tweet_type.isin(['original','reply','quote'])],\n",
    "                  showfliers=False,\n",
    "                  dodge=True,\n",
    "                  ax=axs[i])\n",
    "    \n",
    "    axs[i].set_xlabel(xlb)\n",
    "    axs[i].set_ylabel(\"Sentiment score\")\n",
    "    axs[i].set_title(d+' users',loc='center', pad=pad, fontweight='bold')\n",
    "    i=1\n",
    "\n",
    "fig.tight_layout(pad=2.0)\n",
    "axs[1].get_legend().remove()\n",
    "axs[0].legend(loc='upper right',bbox_to_anchor=(1,1.25), title=\"Party theme\", fancybox=True, ncol=5)\n",
    "#plt.savefig(GRAPHICS_DIR+\"sentiment-boxplots.pdf\", bbox_inches = \"tight\")\n",
    "#plt.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
